{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T15:49:55.584108Z",
     "start_time": "2018-12-27T15:49:31.424419Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import argparse\n",
    "import errno\n",
    "import scipy.misc\n",
    "import dlib\n",
    "import cv2\n",
    "\n",
    "from skimage.feature import hog\n",
    "\n",
    "import tensorflow as tf \n",
    "from tflearn.layers.core import input_data, dropout, fully_connected\n",
    "from tflearn.layers.conv import conv_2d, max_pool_2d\n",
    "from tflearn.layers.merge_ops import merge_outputs, merge\n",
    "from tflearn.layers.normalization import local_response_normalization, batch_normalization\n",
    "from tflearn.layers.estimator import regression \n",
    "from tflearn.optimizers import Momentum, Adam\n",
    "\n",
    "from tflearn import DNN\n",
    "import time\n",
    "\n",
    "import imutils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform datas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T16:46:41.203579Z",
     "start_time": "2018-12-27T16:46:41.198782Z"
    }
   },
   "outputs": [],
   "source": [
    "path = '/Users/maelfabien/filrouge_pole_emploi/Common/'\n",
    "local_path = '/Users/maelfabien/Desktop/LocalDB/Videos/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T16:46:51.271264Z",
     "start_time": "2018-12-27T16:46:42.593254Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Reading the dataset\n",
    "data = pd.read_csv(local_path + 'fer2013.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T16:53:46.602766Z",
     "start_time": "2018-12-27T16:53:42.510335Z"
    }
   },
   "outputs": [],
   "source": [
    "# loading Dlib predictor and preparing arrays:\n",
    "predictor = dlib.shape_predictor('/Users/maelfabien/Desktop/LocalDB/Videos/models/shape_predictor_68_face_landmarks.dat')\n",
    "new_labels = [0, 1, 2, 3, 4, 5, 6]\n",
    "nb_images_per_label = list(np.zeros(len(new_labels), 'uint8'))\n",
    "\n",
    "image_height = 48\n",
    "image_width = 48\n",
    "window_size = 24\n",
    "window_step = 6\n",
    "\n",
    "def get_landmarks(image, rects):\n",
    "    if len(rects) > 1:\n",
    "        raise BaseException(\"TooManyFaces\")\n",
    "    if len(rects) == 0:\n",
    "        raise BaseException(\"NoFaces\")\n",
    "    return np.matrix([[p.x, p.y] for p in predictor(image, rects[0]).parts()])\n",
    "\n",
    "def get_new_label(label, one_hot_encoding=False):\n",
    "    if one_hot_encoding:\n",
    "        new_label = new_labels.index(label)\n",
    "        label = list(np.zeros(len(new_labels), 'uint8'))\n",
    "        label[new_label] = 1\n",
    "        return label\n",
    "    else:\n",
    "        return new_labels.index(label)\n",
    "\n",
    "def sliding_hog_windows(image):\n",
    "    hog_windows = []\n",
    "    for y in range(0, image_height, window_step):\n",
    "        for x in range(0, image_width, window_step):\n",
    "            window = image[y:y+window_size, x:x+window_size]\n",
    "            hog_windows.extend(hog(window, orientations=8, pixels_per_cell=(8, 8),\n",
    "                                            cells_per_block=(1, 1), visualise=False))\n",
    "    return hog_windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T17:31:32.810206Z",
     "start_time": "2018-12-27T16:54:38.835401Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "converting set: Training...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/skimage/feature/_hog.py:150: skimage_deprecation: Default value of `block_norm`==`L1` is deprecated and will be changed to `L2-Hys` in v0.15. To supress this message specify explicitly the normalization method.\n",
      "  skimage_deprecation)\n",
      "/anaconda3/lib/python3.6/site-packages/skimage/feature/_hog.py:248: skimage_deprecation: Argument `visualise` is deprecated and will be changed to `visualize` in v0.16\n",
      "  'be changed to `visualize` in v0.16', skimage_deprecation)\n",
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:31: DeprecationWarning: `imsave` is deprecated!\n",
      "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``imageio.imwrite`` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "converting set: PublicTest...\n",
      "converting set: PrivateTest...\n"
     ]
    }
   ],
   "source": [
    "for category in data['Usage'].unique():\n",
    "    print( \"converting set: \" + category + \"...\")\n",
    "    # create folder\n",
    "    if not os.path.exists(category):\n",
    "        try:\n",
    "            os.makedirs(local_path + \"/model3/\" + category)\n",
    "        except :\n",
    "            pass\n",
    "    \n",
    "    # get samples and labels of the actual category\n",
    "    category_data = data[data['Usage'] == category]\n",
    "    samples = category_data['pixels'].values\n",
    "    labels = category_data['emotion'].values\n",
    "    \n",
    "    # get images and extract features\n",
    "    images = []\n",
    "    labels_list = []\n",
    "    landmarks = []\n",
    "    hog_features = []\n",
    "    hog_images = []\n",
    "    \n",
    "    for i in range(len(samples)):\n",
    "        image = np.fromstring(samples[i], dtype=int, sep=\" \").reshape((image_height, image_width))\n",
    "        images.append(image)\n",
    "        #scipy.misc.imsave(category + '/' + str(i) + '.jpg', image)\n",
    "        \n",
    "        features = sliding_hog_windows(image)\n",
    "        f, hog_image = hog(image, orientations=8, pixels_per_cell=(16, 16),cells_per_block=(1, 1), visualise=True)\n",
    "        hog_features.append(features)\n",
    "        hog_images.append(hog_image)\n",
    "        scipy.misc.imsave('temp.jpg', image)\n",
    "        image2 = cv2.imread('temp.jpg')\n",
    "            \n",
    "        face_rects = [dlib.rectangle(left=1, top=1, right=47, bottom=47)]\n",
    "        face_landmarks = get_landmarks(image2, face_rects)\n",
    "            \n",
    "        landmarks.append(face_landmarks)            \n",
    "        labels_list.append(get_new_label(labels[i], one_hot_encoding=True))\n",
    "            \n",
    "        nb_images_per_label[get_new_label(labels[i])] += 1\n",
    "\n",
    "    np.save(local_path + '/model3/' + category + '/images.npy', images)\n",
    "    np.save(local_path + '/model3/' + category + '/labels.npy', labels_list)\n",
    "    np.save(local_path + '/model3/' + category + '/landmarks.npy', landmarks)\n",
    "    np.save(local_path + '/model3/' + category + '/hog_features.npy', hog_features)\n",
    "    np.save(local_path + '/model3/' + category + '/hog_images.npy', hog_images)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load datas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T17:53:11.334921Z",
     "start_time": "2018-12-27T17:53:11.327683Z"
    }
   },
   "outputs": [],
   "source": [
    "input_size = 48\n",
    "output_size = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T17:43:36.414176Z",
     "start_time": "2018-12-27T17:43:32.229493Z"
    }
   },
   "outputs": [],
   "source": [
    "data_dict = dict()\n",
    "validation_dict = dict()\n",
    "test_dict = dict()\n",
    "\n",
    "# load train set\n",
    "data_dict['X'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/Training/images.npy')\n",
    "data_dict['X'] = data_dict['X'].reshape([-1, input_size, input_size, 1])\n",
    "data_dict['X2'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/Training/landmarks.npy')\n",
    "data_dict['X2'] = np.array([x.flatten() for x in data_dict['X2']])\n",
    "data_dict['X2'] = np.concatenate((data_dict['X2'], np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/Training/hog_features.npy')), axis=1)\n",
    "data_dict['Y'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/Training/labels.npy')\n",
    "\n",
    "test_dict['X'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PublicTest/images.npy')\n",
    "test_dict['X'] = test_dict['X'].reshape([-1, input_size, input_size, 1])\n",
    "test_dict['X2'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PublicTest/landmarks.npy')\n",
    "test_dict['X2'] = np.array([x.flatten() for x in test_dict['X2']])\n",
    "test_dict['X2'] = np.concatenate((test_dict['X2'], np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PublicTest/hog_features.npy')), axis=1)\n",
    "test_dict['Y'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PublicTest/labels.npy')\n",
    "  \n",
    "validation_dict['X'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PrivateTest/images.npy')\n",
    "validation_dict['X'] = validation_dict['X'].reshape([-1, input_size, input_size, 1])\n",
    "validation_dict['X2'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PrivateTest/landmarks.npy')\n",
    "validation_dict['X2'] = np.array([x.flatten() for x in validation_dict['X2']])\n",
    "validation_dict['X2'] = np.concatenate((validation_dict['X2'], np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PrivateTest/hog_features.npy')), axis=1)\n",
    "validation_dict['Y'] = np.load('/Users/maelfabien/Desktop/LocalDB/Videos/model3/PrivateTest/labels.npy')       "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T17:59:50.369602Z",
     "start_time": "2018-12-27T17:59:50.314478Z"
    }
   },
   "outputs": [],
   "source": [
    "def build_modelB() :\n",
    "    \n",
    "    images_network = input_data(shape=[None, input_size, input_size, 1], name='input1')\n",
    "    images_network = conv_2d(images_network, 64, 3, activation='relu')\n",
    "    images_network = batch_normalization(images_network)\n",
    "    \n",
    "    images_network = max_pool_2d(images_network, 3, strides = 2)\n",
    "    images_network = conv_2d(images_network, 128, 3, activation='relu')\n",
    "    images_network = batch_normalization(images_network)\n",
    "    \n",
    "    images_network = max_pool_2d(images_network, 3, strides = 2)\n",
    "    images_network = conv_2d(images_network, 256, 3, activation='relu')\n",
    "    images_network = batch_normalization(images_network)\n",
    "    \n",
    "    images_network = max_pool_2d(images_network, 3, strides = 2)\n",
    "    images_network = dropout(images_network, keep_prob=True)\n",
    "    \n",
    "    images_network = fully_connected(images_network, 4096, activation='relu')\n",
    "    images_network = dropout(images_network, keep_prob=0.956)\n",
    "    images_network = fully_connected(images_network, 1024, activation='relu')\n",
    "    \n",
    "    landmarks_network = input_data(shape=[None, 2728], name='input2')\n",
    "    landmarks_network = fully_connected(landmarks_network, 1024, activation='relu')\n",
    "    landmarks_network = batch_normalization(landmarks_network)\n",
    "    \n",
    "    images_network = fully_connected(images_network, 128, activation='relu')\n",
    "    network = merge([images_network, landmarks_network], 'concat', axis=1)\n",
    "    \n",
    "    network = fully_connected(network, output_size, activation='softmax')\n",
    "\n",
    "    optimizer = Momentum(learning_rate=0.016, momentum=0.95,lr_decay=0.864, decay_step=50)\n",
    "    network = regression(network, optimizer=optimizer, loss='categorical_crossentropy', learning_rate=0.016, name='output')\n",
    "\n",
    "    return network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-12-27T19:11:10.003Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Step: 517  | total loss: \u001b[1m\u001b[32m1.22977\u001b[0m\u001b[0m | time: 306.509s\n",
      "\u001b[2K\r",
      "| Momentum | epoch: 003 | loss: 1.22977 - acc: 0.5357 -- iter: 08576/28709\n"
     ]
    }
   ],
   "source": [
    "with tf.Graph().as_default():\n",
    "    data = data_dict\n",
    "    validation = validation_dict\n",
    "    test = test_dict\n",
    "    print( \"building model...\")\n",
    "    network = build_modelB()\n",
    "    model = DNN(network, tensorboard_dir=\"/Users/maelfabien/Desktop/LocalDB/Videos/model3/logs\", tensorboard_verbose=0, \n",
    "                checkpoint_path=\"/Users/maelfabien/Desktop/LocalDB/Videos/model3/checkpoint/chk\",max_checkpoints=1)\n",
    "\n",
    "    start_time = time.time()\n",
    "    model.fit([data['X'], data['X2']], data['Y'],\n",
    "                validation_set=([validation['X'], validation['X2']], validation['Y']),\n",
    "                snapshot_step=500,show_metric=True,batch_size=128,n_epoch=13)\n",
    "    validation['X2'] = None\n",
    "    training_time = time.time() - start_time\n",
    "    print( \"training time = {0:.1f} sec\".format(training_time))\n",
    "\n",
    "    print( \"saving model...\")\n",
    "    model.save(\"/Users/maelfabien/Desktop/LocalDB/Videos/model3/models/saved_model.bin\")\n",
    "\n",
    "    print( \"evaluating...\")\n",
    "    validation_accuracy = evaluate(model, validation['X'], validation['X2'], validation['Y'])\n",
    "    print( \"  - validation accuracy = {0:.1f}\".format(validation_accuracy*100))\n",
    "    print(validation_accuracy)\n",
    "\n",
    "def evaluate(model, X, X2, Y):\n",
    "    accuracy = model.evaluate([X, X2], Y)\n",
    "    return accuracy[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 24\n",
    "window_step = 6\n",
    "\n",
    "def load_model():\n",
    "    model = None\n",
    "    with tf.Graph().as_default():\n",
    "        print( \"loading pretrained model...\")\n",
    "        network = build_model()\n",
    "        model = DNN(network)\n",
    "        model.load(\"/Users/maelfabien/Desktop/LocalDB/Videos/model3/models\")\n",
    "    return model\n",
    "\n",
    "def get_landmarks(image, rects, predictor):\n",
    "    # this function have been copied from http://bit.ly/2cj7Fpq\n",
    "    if len(rects) > 1:\n",
    "        break\n",
    "        #raise TooManyFaces\n",
    "    if len(rects) == 0:\n",
    "        break\n",
    "        #raise NoFaces\n",
    "    return np.matrix([[p.x, p.y] for p in predictor(image, rects[0]).parts()])\n",
    "\n",
    "def sliding_hog_windows(image):\n",
    "    hog_windows = []\n",
    "    for y in range(0, input_size, window_step):\n",
    "        for x in range(0, input_size, window_step):\n",
    "            window = image[y:y+window_size, x:x+window_size]\n",
    "            hog_windows.extend(hog(window, orientations=8, pixels_per_cell=(8, 8),\n",
    "                                            cells_per_block=(1, 1), visualise=False))\n",
    "    return hog_windows\n",
    "\n",
    "def predict(image, model, shape_predictor=None):\n",
    "    \n",
    "    # get landmarks\n",
    "    face_rects = [dlib.rectangle(left=0, top=0, right=input_size, bottom=input_size)]\n",
    "    face_landmarks = np.array([get_landmarks(image, face_rects, shape_predictor)])\n",
    "    features = face_landmarks\n",
    "    \n",
    "    hog_features = sliding_hog_windows(image)\n",
    "    hog_features = np.asarray(hog_features)\n",
    "    \n",
    "    face_landmarks = face_landmarks.flatten()\n",
    "    features = np.concatenate((face_landmarks, hog_features))\n",
    "     \n",
    "    tensor_image = image.reshape([-1, input_size, input_size, 1])\n",
    "    predicted_label = model.predict([tensor_image, features.reshape((1, -1))])\n",
    "    \n",
    "    return get_emotion(predicted_label[0])\n",
    "\n",
    "\n",
    "def get_emotion(label):\n",
    "    print( \"- Angry: {0:.1f}%\\n- Happy: {1:.1f}%\\n- Sad: {2:.1f}%\\n- Surprise: {3:.1f}%\\n- Neutral: {4:.1f}%\".format(\n",
    "                label[0]*100, label[1]*100, label[2]*100, label[3]*100, label[4]*100))\n",
    "    label = label.tolist()\n",
    "    emotions = [\"Angry\", \"Disgust\", \"Fear\", \"Happy\", \"Sad\", \"Surprise\", \"Neutral\"]\n",
    "    return emotions[label.index(max(label))], max(label)\n",
    "\n",
    "# parse arg to see if we need to launch training now or not yet\n",
    "model = load_model()\n",
    "hanks = '/Users/maelfabien/filrouge_pole_emploi/Common/test_samples/tom-hanks.jpg'\n",
    "\n",
    "image = cv2.imread(hanks, 0)\n",
    "start_time = time.time()\n",
    "\n",
    "emotion, confidence = predict(image, model, predictor)\n",
    "total_time = time.time() - start_time\n",
    "\n",
    "print( \"Prediction: {0} (confidence: {1:.1f}%)\".format(emotion, confidence*100))\n",
    "print( \"time: {0:.1f} sec\".format(total_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict from video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOX_COLOR = (0, 255, 0)\n",
    "TEXT_COLOR = (0, 255, 0)\n",
    "\n",
    "# initializebevideo stream\n",
    "video_stream = cv2.VideoCapture(0)\n",
    "face_detector = cv2.CascadeClassifier('/Users/maelfabien/Desktop/LocalDB/Videos/models/lbpcascade_frontalface.xml')\n",
    "\n",
    "shape_predictor = dlib.shape_predictor('/Users/maelfabien/Desktop/LocalDB/Videos/models/shape_predictor_68_face_landmarks.dat')\n",
    "        \n",
    "model = load_model()\n",
    "last_predicted_time = 0\n",
    "last_predicted_confidence = 0\n",
    "last_predicted_emotion = \"\"\n",
    "\n",
    "def predict_emotion(image):\n",
    "    image.resize([input_size, input_size], refcheck=False)\n",
    "    emotion, confidence = predict(image, model, shape_predictor)\n",
    "    return emotion, confidence\n",
    "\n",
    "def recognize_emotions():\n",
    "    failedFramesCount = 0\n",
    "    detected_faces = []\n",
    "    time_last_sent = 0\n",
    "    \n",
    "    while True:\n",
    "        grabbed, frame = video_stream.read()\n",
    "\n",
    "        if grabbed:\n",
    "        # detection phase\n",
    "            frame = imutils.resize(frame, width=600)\n",
    "            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "            # detect faces\n",
    "            faces = face_detector.detectMultiScale(gray, 1.3, 5)\n",
    "            for (x,y,w,h) in faces:\n",
    "                if w < 30 and h<30: # skip the small faces (probably false detections)\n",
    "                    continue\n",
    "\n",
    "                # bounding box\n",
    "                cv2.rectangle(frame, (x, y), (x + w, y + h), BOX_COLOR, 2)\n",
    "\n",
    "                # try to recognize emotion\n",
    "                face = gray[y:y+h, x:x+w].copy()\n",
    "                \n",
    "                if time.time() - self.last_predicted_time < 0.5 :\n",
    "                    label = last_predicted_emotion\n",
    "                    confidence = self.last_predicted_confidence\n",
    "                else:\n",
    "                    label, confidence = predict_emotion(face)\n",
    "                    last_predicted_emotion = label\n",
    "                    last_predicted_confidence = confidence\n",
    "                    last_predicted_time = time.time()\n",
    "                    \n",
    "                # display and send message by socket\n",
    "                text = \"{0} ({1:.1f}%)\".format(label, confidence*100)\n",
    "                if label is not None:\n",
    "                    cv2.putText(frame, text, (x - 20, y - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, TEXT_COLOR, 2)\n",
    "\n",
    "            # display images\n",
    "            cv2.imshow(\"Facial Expression Recognition\", frame)\n",
    "\n",
    "            key = cv2.waitKey(1) & 0xFF\n",
    "            if key == ord(\"q\"):\n",
    "                break            \n",
    "        else:\n",
    "            failedFramesCount += 1\n",
    "            if failedFramesCount > 10:\n",
    "                print( \"can't grab frames\")\n",
    "                break\n",
    "\n",
    "    self.video_stream.release()\n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "r = EmotionRecognizer()\n",
    "r.recognize_emotions()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "class EmotionRecognizer:\n",
    "    \n",
    "    BOX_COLOR = (0, 255, 0)\n",
    "    TEXT_COLOR = (0, 255, 0)\n",
    "\n",
    "    def __init__(self):\n",
    "       \n",
    "        # initializebevideo stream\n",
    "        self.video_stream = cv2.VideoCapture(VIDEO_PREDICTOR.camera_source)\n",
    "  \n",
    "        self.face_detector = cv2.CascadeClassifier(VIDEO_PREDICTOR.face_detection_classifier)\n",
    "\n",
    "        self.shape_predictor = None\n",
    "        if NETWORK.use_landmarks:\n",
    "            self.shape_predictor = dlib.shape_predictor(DATASET.shape_predictor_path)\n",
    "        \n",
    "        self.model = load_model()\n",
    "        self.last_predicted_time = 0\n",
    "        self.last_predicted_confidence = 0\n",
    "        self.last_predicted_emotion = \"\"\n",
    "\n",
    "    def predict_emotion(self, image):\n",
    "        image.resize([NETWORK.input_size, NETWORK.input_size], refcheck=False)\n",
    "        emotion, confidence = predict(image, self.model, self.shape_predictor)\n",
    "        return emotion, confidence\n",
    "\n",
    "    def recognize_emotions(self):\n",
    "        failedFramesCount = 0\n",
    "        detected_faces = []\n",
    "        time_last_sent = 0\n",
    "        while True:\n",
    "            grabbed, frame = self.video_stream.read()\n",
    "\n",
    "            if grabbed:\n",
    "                # detection phase\n",
    "                frame = imutils.resize(frame, width=600)\n",
    "                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "                # detect faces\n",
    "                faces = self.face_detector.detectMultiScale(gray, 1.3, 5)\n",
    "                for (x,y,w,h) in faces:\n",
    "                    if w < 30 and h<30: # skip the small faces (probably false detections)\n",
    "                        continue\n",
    "\n",
    "                    # bounding box\n",
    "                    cv2.rectangle(frame, (x, y), (x + w, y + h), self.BOX_COLOR, 2)\n",
    "\n",
    "                    # try to recognize emotion\n",
    "                    face = gray[y:y+h, x:x+w].copy()\n",
    "                    if time.time() - self.last_predicted_time < VIDEO_PREDICTOR.time_to_wait_between_predictions:\n",
    "                        label = self.last_predicted_emotion\n",
    "                        confidence = self.last_predicted_confidence\n",
    "                    else:\n",
    "                        label, confidence = self.predict_emotion(face)\n",
    "                        self.last_predicted_emotion = label\n",
    "                        self.last_predicted_confidence = confidence\n",
    "                        self.last_predicted_time = time.time()\n",
    "                    \n",
    "                    # display and send message by socket\n",
    "                    if VIDEO_PREDICTOR.show_confidence:\n",
    "                        text = \"{0} ({1:.1f}%)\".format(label, confidence*100)\n",
    "                    else:\n",
    "                        text = label\n",
    "                    if label is not None:\n",
    "                        cv2.putText(frame, text, (x - 20, y - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.TEXT_COLOR, 2)\n",
    "\n",
    "                # display images\n",
    "                cv2.imshow(\"Facial Expression Recognition\", frame)\n",
    "\n",
    "                key = cv2.waitKey(1) & 0xFF\n",
    "                if key == ord(\"q\"):\n",
    "                    break            \n",
    "            else:\n",
    "                failedFramesCount += 1\n",
    "                if failedFramesCount > 10:\n",
    "                    print( \"can't grab frames\")\n",
    "                    break\n",
    "\n",
    "        self.video_stream.release()\n",
    "        cv2.destroyAllWindows()\n",
    "\n",
    "r = EmotionRecognizer()\n",
    "r.recognize_emotions()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
